{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "from sklearn.preprocessing import LabelEncoder,OneHotEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer,HashingVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD,SparsePCA\n",
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score,roc_auc_score,f1_score,recall_score\n",
    "\n",
    "import gc\n",
    "import time\n",
    "import os\n",
    "import sys\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "pickle_path = \"../pickle\"\n",
    "\n",
    "train = pd.read_csv(\"../data/age_train.csv\",names=['uid','age_group']).sort_values(by=['uid'])\n",
    "test = pd.read_csv(\"../data/age_test.csv\",names=['uid']).sort_values(by=['uid'])\n",
    "active = pd.read_pickle(\"{}/user_app_active.pickle\".format(pickle_path))\n",
    "usage_appid_seq = pd.read_pickle(\"{}/user_app_seq.pickle\".format(pickle_path))\n",
    "# print((train.shape,test.shape),(info.shape,active.shape,user_basic_info.shape,behavior_info.shape,usage.shape))\n",
    "\n",
    "all_data = train.append(test)\n",
    "all_data = all_data.sort_values(by=['uid']).reset_index(drop=True)\n",
    "print(all_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       uid                                              appid  app_len  \\\n",
      "0  1000006  a001012 a001036 a001062 a001172 a001275 a00135...     47.0   \n",
      "1  1000009  a001012 a001015 a001055 a001062 a00107 a001072...     73.0   \n",
      "2  1000010  a001012 a001036 a001050 a001055 a001062 a00107...     96.0   \n",
      "3  1000011  a001012 a001063 a002450 a003083 a00326 a003987...     21.0   \n",
      "4  1000012  a001036 a001062 a001580 a001583 a003570 a00365...     33.0   \n",
      "\n",
      "   age_group  \n",
      "0        4.0  \n",
      "1        4.0  \n",
      "2        5.0  \n",
      "3        NaN  \n",
      "4        5.0  \n",
      "TFIDF & COUNT FINISHED...\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer,HashingVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD,SparsePCA\n",
    "from sklearn.linear_model import LogisticRegression,BayesianRidge,SGDClassifier,PassiveAggressiveClassifier,RidgeClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB, MultinomialNB\n",
    "from sklearn.ensemble import ExtraTreesClassifier,RandomForestClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import LinearSVC,NuSVC,SVC\n",
    "from sklearn.metrics import roc_auc_score,accuracy_score\n",
    "from sklearn.model_selection import KFold,StratifiedKFold,TimeSeriesSplit\n",
    "from scipy import sparse\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "import catboost as cbt\n",
    "\n",
    "def get_sklearn_embedding(now,n_splits=5,ngram=1,prefix=None):\n",
    "    \n",
    "    if os.path.exists(\"../pickle/{}_tfidf_count_emb_all.pickle\".format(prefix)):\n",
    "        return pd.read_pickle(\"../pickle/{}_tfidf_count_emb_all.pickle\".format(prefix))\n",
    "    else:\n",
    "        df = now.copy()\n",
    "        df['appid'] = df['appid'].map(lambda x:\" \".join(x))\n",
    "        df = df.merge(all_data,how='right',on='uid')\n",
    "        print(df.head())\n",
    "        tfidf = TfidfVectorizer(ngram_range=(1,ngram))\n",
    "        tf = tfidf.fit_transform(df['appid'].fillna(\"##\").values)\n",
    "        count = CountVectorizer(ngram_range=(1,ngram))\n",
    "        cv = count.fit_transform(df['appid'].fillna(\"##\").values)\n",
    "        all_ = sparse.csr_matrix(sparse.hstack([tf, cv]))\n",
    "        print(\"TFIDF & COUNT FINISHED...\")\n",
    "        tr = df['age_group'].notnull()\n",
    "        te = df['age_group'].isnull()\n",
    "        y = df[tr]['age_group']-1\n",
    "        X_train = all_[df[tr].index]\n",
    "        X_test = all_[df[te].index]\n",
    "\n",
    "        random_seed = 2019\n",
    "        model_zoo = [SGDClassifier(n_jobs=10,verbose=1),SGDClassifier(loss='log',n_jobs=10,verbose=1),\n",
    "                     SGDClassifier(loss='modified_huber',n_jobs=10,verbose=1),\n",
    "                     PassiveAggressiveClassifier(n_jobs=10,verbose=1),LogisticRegression(C=10),\n",
    "                     RidgeClassifier(solver='lsqr',fit_intercept=False),LinearSVC(verbose=1,max_iter=500),\n",
    "                     BernoulliNB(),MultinomialNB()]\n",
    "\n",
    "        columns = ['SGD_HINGE','SGD_LOG','SGD_HUBER','PAC','LR','RIDGE','LSVC','BNB','MNB']\n",
    "\n",
    "        oof = []\n",
    "        count = 0\n",
    "\n",
    "        for model in model_zoo:\n",
    "            t1 = time.time()\n",
    "            cv_pred_stack = np.zeros((X_train.shape[0],num_classes))\n",
    "            test_pred_stack = np.zeros((X_test.shape[0],num_classes))\n",
    "            skf = KFold(n_splits=n_splits,random_state=random_seed)\n",
    "            if os.path.exists(\"../pickle/{}_TFIDF_COUNT_{}.pickle\".format(prefix,columns[count])):\n",
    "                tmp = pd.read_pickle(\"../pickle/{}_TFIDF_COUNT_{}.pickle\".format(prefix,columns[count]))\n",
    "            else:\n",
    "                for index, (train_index, test_index) in enumerate(skf.split(X_train, y)):\n",
    "                    print(index,model)\n",
    "                    train_x, test_x, train_y, test_y = X_train[train_index], X_train[test_index], y.iloc[train_index], y.iloc[test_index]\n",
    "                    model.fit(train_x,train_y)\n",
    "                    try:\n",
    "                        y_val = model._predict_proba_lr(test_x)\n",
    "                    except:\n",
    "                        y_val = model.predict_proba(test_x)\n",
    "                    cv_pred_stack[test_index] = y_val\n",
    "                    print(y_val.shape)\n",
    "                    try:\n",
    "                        test_pred_stack += model._predict_proba_lr(X_test) / n_splits\n",
    "                    except:\n",
    "                        test_pred_stack += model.predict_proba(X_test) / n_splits\n",
    "                print(model,'score:',accuracy_score(y,np.argmax(cv_pred_stack,axis=1)))\n",
    "                print(time.time()-t1)\n",
    "                a = pd.DataFrame(cv_pred_stack).add_prefix(columns[count]+\"_\")\n",
    "                a['uid'] = df[tr]['uid'].values\n",
    "                b = pd.DataFrame(test_pred_stack).add_prefix(columns[count]+\"_\")\n",
    "                b['uid'] = df[te]['uid'].values\n",
    "                tmp = a.append(b).sort_values(by=['uid']).reset_index(drop=True)\n",
    "                tmp.to_pickle(\"../pickle/{}_TFIDF_COUNT_{}.pickle\".format(prefix,columns[count]))\n",
    "                \n",
    "            count += 1\n",
    "            oof.append(tmp)\n",
    "  \n",
    "        df_agg = pd.DataFrame()\n",
    "        for i in tqdm(oof):\n",
    "            df_agg[i.columns] = i\n",
    "        df_agg = df_agg.sort_values(by=['uid'],ascending=True)\n",
    "        df_agg.to_pickle(\"../pickle/{}_tfidf_count_emb_all.pickle\".format(prefix))\n",
    "    \n",
    "    return df_agg\n",
    "\n",
    "num_classes = 6\n",
    "prob_active = get_sklearn_embedding(active,n_splits=5,ngram=1,prefix='active')\n",
    "prob_usage = get_sklearn_embedding(usage_appid_seq,n_splits=5,ngram=1,prefix='usage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
